# -*- coding:utf-8 -*-
"""
针对sinahotnews3数据库！！！

用来统计每个股票每天有多少新闻，新闻数目就可以看作是今天该新闻的热度

naiveHotFactor.py
对于每一条新闻，获取该新闻对应的股票列表，发布日期
1.对于日期建立dict，可以快速定位到date
2.在date下面对stock建立dict，可以快速定位到stock
3.结合上述两步，每次对相应日期相应股票进行+1操作，最终按照日期来记录每个stock的出现次数即可
4.只需要遍历一次数据库，便捷高效

WeightedHotFactor.py
相较于上一个版本，在这一个版本中考虑了一级映射与二级映射
1.对于新闻中直接出现的股票，出现一次记为+1
2.对于新闻中没有直接提及的股票，出现一次记为+weight
author: yinzm
"""
import pymongo
from datetime import datetime
import pandas as pd
import os
import pickle

weight = input("input weight: ")
weight = float(weight)

mongoClient = pymongo.MongoClient("localhost", 27017)
db = mongoClient.sinafinance
collection = db.sinahotnews3 # 别写错了

def dateList(beginDate, endDate):
    # 产生所有的日期, beginDate: str, endDate: str
    dateSimple = [datetime.strftime(x,"%Y%m%d") for x in list(pd.date_range(start=beginDate, end=endDate))]
    return dateSimple

allDateList = dateList("20130101", "20180331")

# 按照日期将所有的映射关系加载进来
date_wind = []
for year in range(2012, 2018):
    for date_str in ['0501', '0901', '1101']:
        date_wind.append(str(year)+date_str)

pklPath = '../DataPreprocess/ExtractWindInfo/data/pklFiles'

relation = {} # relation['20120501'] = {'boardchairman':boardchairman, 'ceo':ceo, ....}
for date in date_wind:
    filePath = os.path.join(pklPath, date)
    relation[date] = {}
    boardchairman = pickle.load(open(os.path.join(filePath, 'boardchairman.pkl'), 'rb'))
    relation[date]['boardchairman'] = boardchairman
    ceo = pickle.load(open(os.path.join(filePath, 'ceo.pkl'), 'rb'))
    relation[date]['ceo'] = ceo
    chairman = pickle.load(open(os.path.join(filePath, 'chairman.pkl'), 'rb'))
    relation[date]['chairman'] = chairman
    compName = pickle.load(open(os.path.join(filePath, 'compName.pkl'), 'rb'))
    relation[date]['compName'] = compName
    holderController = pickle.load(open(os.path.join(filePath, 'holderController.pkl'), 'rb'))
    relation[date]['holderController'] = holderController
    holderLiqname = pickle.load(open(os.path.join(filePath, 'holderLiqname.pkl'), 'rb'))
    relation[date]['holderLiqname'] = holderLiqname
    holderName = pickle.load(open(os.path.join(filePath, 'holderName.pkl'), 'rb'))
    relation[date]['holderName'] = holderName
    preName = pickle.load(open(os.path.join(filePath, 'preName.pkl'), 'rb'))
    relation[date]['preName'] = preName
    secName = pickle.load(open(os.path.join(filePath, 'secName.pkl'), 'rb'))
    relation[date]['secName'] = secName
    secName1 = pickle.load(open(os.path.join(filePath, 'secName1.pkl'), 'rb'))
    relation[date]['secName1'] = secName1
    stockCodeDiffVariation = pickle.load(open(os.path.join(filePath, 'stockCodeDiffVariation.pkl'), 'rb'))
    relation[date]['stockCodeDiffVariation'] = stockCodeDiffVariation

# 加载所有的股票列表
stockCodeList = []
with open('../DataPreprocess/ExtractWindInfo/data/stockList.csv', 'r', encoding='utf-8') as f:
    for stock in f.readlines():
        stockCode = stock.strip()[:-3]
        stockCodeList.append(stockCode)

# 初始化
dateDict = dict()
for d in allDateList:
    dateDict[d] = dict()
    for s in stockCodeList:
        dateDict[d][s] = 0

# 开始遍历数据库
cnt = 0
for item in collection.find():
    date = str(item['date'])
    # 找到该新闻所能用到的最新的relation
    relation_date = None
    for i in range(1, len(date_wind)):
        if date > date_wind[i - 1] and date <= date_wind[i]:
            relation_date = date_wind[i - 1]
            break
    if relation_date == None:
        continue
    compName = relation[relation_date]['compName']
    preName = relation[relation_date]['preName']
    secName = relation[relation_date]['secName']
    secName1 = relation[relation_date]['secName1']
    stockCodeDiffVariation = relation[relation_date]['stockCodeDiffVariation']
    boardchairman = relation[relation_date]['boardchairman']
    ceo = relation[relation_date]['ceo']
    chairman = relation[relation_date]['chairman']
    holderController = relation[relation_date]['holderController']
    holderLiqname = relation[relation_date]['holderLiqname']
    holderName = relation[relation_date]['holderName']

    new = item['text'].strip().split('/')
    tempDict = {}
    for word in new:
        # 确定word映射的stock code
        # 并且要确定stock code的权重
        if word in stockCodeList:
            if word not in tempDict:
                tempDict[word] = 1
            else:
                tempDict[word] = max(1, tempDict[word])
        # 下面这部分都是通过股票名字或者代码映射到的，属于一级映射
        elif word in compName:
            scode = compName[word]
            if scode not in tempDict:
                tempDict[scode] = 1
            else:
                tempDict[scode] = max(1, tempDict[scode])
        elif word in preName:
            scode = preName[word]
            if scode not in tempDict:
                tempDict[scode] = 1
            else:
                tempDict[scode] = max(1, tempDict[scode])
        elif word in secName:
            scode = secName[word]
            if scode not in tempDict:
                tempDict[scode] = 1
            else:
                tempDict[scode] = max(1, tempDict[scode])
        elif word in secName1:
            scode = secName1[word]
            if scode not in tempDict:
                tempDict[scode] = 1
            else:
                tempDict[scode] = max(1, tempDict[scode])
        elif word in stockCodeDiffVariation:
            scode = stockCodeDiffVariation[word]
            if scode not in tempDict:
                tempDict[scode] = 1
            else:
                tempDict[scode] = max(1, tempDict[scode])
        # 下面这部分都是通过股票的持有者或者管理者映射到的，属于二级映射
        elif word in boardchairman:
            scode = boardchairman[word]
            if word not in tempDict:
                tempDict[scode] = weight
            else:
                tempDict[scode] = max(weight, tempDict[scode])
        elif word in ceo:
            scode = ceo[word]
            if word not in tempDict:
                tempDict[scode] = weight
            else:
                tempDict[scode] = max(weight, tempDict[scode])
        elif word in chairman:
            scode = chairman[word]
            if word not in tempDict:
                tempDict[scode] = weight
            else:
                tempDict[scode] = max(weight, tempDict[scode])
        elif word in holderController:
            scode = holderController[word]
            if word not in tempDict:
                tempDict[scode] = weight
            else:
                tempDict[scode] = max(weight, tempDict[scode])
        elif word in holderLiqname:
            scode = holderLiqname[word]
            if word not in tempDict:
                tempDict[scode] = weight
            else:
                tempDict[scode] = max(weight, tempDict[scode])
        elif word in holderName:
            scode = holderName[word]
            if word not in tempDict:
                tempDict[scode] = weight
            else:
                tempDict[scode] = max(weight, tempDict[scode])

    # 将得到的股票权重进行登记
    for stock in tempDict:
        if stock in stockCodeList and date in allDateList:
            dateDict[date][stock] += tempDict[stock]

    cnt += 1
    if cnt%100 == 0:
        print(cnt)

# 保存结果
if weight == 0.0:
    weight = int(0)
dateFile = 'E:\\yinzm\\nlp\\data\\Wu_HotFactorData_'+str(weight*100)[:2]
for d in allDateList:
    outputFile = os.path.join(dateFile, d+'.csv')
    with open(outputFile, 'w', encoding='utf-8') as f:
        for stock in stockCodeList:
            f.write("%s,%f\n" % (stock,dateDict[d][stock]))

print("==============over===================")
